Libraries and Data

library(brms)
library(rstan)
library(tidyverse)
library(cowplot)
library(colorblindr)
library(ggpubr)
library(tidybayes)
library(modelr)

# recommended code to speed up stan
# see https://github.com/stan-dev/rstan/wiki/RStan-Getting-Started
rstan_options(auto_write = TRUE)
options(mc.cores = parallel::detectCores())
vis_display_order <- c("point","barchart","interval","table","hopsdist","dotplot","hops","density")
max_eval = 30

rq3 <- read_csv("data/rq3.csv") %>%
    mutate(
    eval_period_normalized = ((evalPeriod - max_eval) / max_eval) + 0.5
  )
rq3 %>%
  ggplot(aes(x = evalPeriod, y = dv, color = treatment)) +
  stat_summary(fun.data = mean_se) +
  geom_hline(yintercept = 1) +
  stat_smooth(method = lm) +
  facet_grid(.~treatment) +
  theme(legend.position = "none")

Priors

Commented out.

get_prior(bf(
    dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken), 
    phi ~ eval_period_normalized * treatment
  ),
  data = rq3, family = Beta)
##                 prior     class                                     coef
##                (flat)         b                                         
##                (flat)         b                   eval_period_normalized
##                (flat)         b  eval_period_normalized:treatmentdensity
##                (flat)         b  eval_period_normalized:treatmentdotplot
##                (flat)         b     eval_period_normalized:treatmenthops
##                (flat)         b eval_period_normalized:treatmenthopsdist
##                (flat)         b eval_period_normalized:treatmentinterval
##                (flat)         b    eval_period_normalized:treatmentpoint
##                (flat)         b    eval_period_normalized:treatmenttable
##                (flat)         b                         treatmentdensity
##                (flat)         b                         treatmentdotplot
##                (flat)         b                            treatmenthops
##                (flat)         b                        treatmenthopsdist
##                (flat)         b                        treatmentinterval
##                (flat)         b                           treatmentpoint
##                (flat)         b                           treatmenttable
##  student_t(3, 0, 2.5) Intercept                                         
##  student_t(3, 0, 2.5)        sd                                         
##  student_t(3, 0, 2.5)        sd                                         
##  student_t(3, 0, 2.5)        sd                                Intercept
##                (flat)         b                                         
##                (flat)         b                   eval_period_normalized
##                (flat)         b  eval_period_normalized:treatmentdensity
##                (flat)         b  eval_period_normalized:treatmentdotplot
##                (flat)         b     eval_period_normalized:treatmenthops
##                (flat)         b eval_period_normalized:treatmenthopsdist
##                (flat)         b eval_period_normalized:treatmentinterval
##                (flat)         b    eval_period_normalized:treatmentpoint
##                (flat)         b    eval_period_normalized:treatmenttable
##                (flat)         b                         treatmentdensity
##                (flat)         b                         treatmentdotplot
##                (flat)         b                            treatmenthops
##                (flat)         b                        treatmenthopsdist
##                (flat)         b                        treatmentinterval
##                (flat)         b                           treatmentpoint
##                (flat)         b                           treatmenttable
##  student_t(3, 0, 2.5) Intercept                                         
##      group resp dpar nlpar bound       source
##                                       default
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                  (vectorized)
##                                       default
##                                       default
##  usertoken                       (vectorized)
##  usertoken                       (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi             (vectorized)
##                  phi                  default
pr_beta = c(
  prior(normal(0, 1), class = b),
  # these prior intercepts are wide and cover 0 (50% on the logit scale), but
  # also assume some likely better-than-50% performance on average --- this
  # was chosen to aid convergence during the pilot, but does not have a strong
  # impact on final estimates.
  prior(normal(2, 2), class = Intercept),
  prior(normal(2, 2), class = Intercept, dpar = phi),
  prior(normal(0, 1), class = b, dpar = phi),
  prior(student_t(3, 0, 1), class = sd)
)

Fit model

# save model - core model
   # dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken), 
   #  phi ~ eval_period_normalized * treatment),
mbeta = brm(bf(
    dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken), 
    phi ~ eval_period_normalized * treatment),
  data = rq3, 
  prior = pr_beta, 
  control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
  warmup = warmup, iter = iter, thin = thin,
  family = Beta
  )


save(mbeta, file = "models/fit1.rda")
mbeta2 = brm(bf(
    dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken), 
    phi ~ eval_period_normalized * treatment),
  data = rq3, 
  prior = pr_beta, 
  control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
  warmup = warmup, iter = iter, thin = thin,
  family = Beta
  )   #  phi ~ eval_period_normalized ),
# challenger: phi eval_period_nor
save(mbeta2, file = "models/fit2.rda")
mbeta3 = brm(bf(
    dv ~ eval_period_normalized + treatment + (1|usertoken), 
    phi ~ eval_period_normalized),
  data = rq3, 
  prior = pr_beta, 
  control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
  warmup = warmup, iter = iter, thin = thin,
  family = Beta
  )   #  phi ~ eval_period_normalized ),
# challenger: phi eval_period_nor
save(mbeta3, file = "models/fit3.rda")
# save model - core model
   # dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken), 
   #  phi ~ eval_period_normalized * treatment),
mbeta4 = brm(bf(
    dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken), 
    phi ~ eval_period_normalized * treatment),
  data = rq3, 
  #prior = pr_beta, 
  control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
  warmup = warmup, iter = iter, thin = thin,
  family = Beta
  )


save(mbeta4, file = "models/fit4.rda")
mbeta5 = brm(bf(
    dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1|usertoken), 
    phi ~ eval_period_normalized * treatment),
  data = rq3, 
  #prior = pr_beta, 
  control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
  warmup = warmup, iter = iter, thin = thin,
  family = Beta
  )   #  phi ~ eval_period_normalized ),
# challenger: phi eval_period_nor
save(mbeta5, file = "models/fit5.rda")
mbeta6 = brm(bf(
    dv ~ eval_period_normalized + treatment + (1|usertoken), 
    phi ~ eval_period_normalized),
  data = rq3, 
  #prior = pr_beta, 
  control = list(adapt_delta = 0.9995, max_treedepth = 15, stepsize = 0.005),
  warmup = warmup, iter = iter, thin = thin,
  family = Beta
  )   #  phi ~ eval_period_normalized ),
# challenger: phi eval_period_nor
save(mbeta6, file = "models/fit6.rda")
# load model
load("models/fit1.rda")
load("models/fit2.rda")
load("models/fit3.rda")
load("models/fit4.rda")
load("models/fit5.rda")
load("models/fit6.rda")

Model results

summary(mbeta)
##  Family: beta 
##   Links: mu = logit; phi = log 
## Formula: dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1 | usertoken) 
##          phi ~ eval_period_normalized * treatment
##    Data: rq3 (Number of observations: 1393) 
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~usertoken (Number of levels: 199) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.63      0.04     0.55     0.72 1.00     1946     3028
## 
## Population-Level Effects: 
##                                              Estimate Est.Error l-95% CI
## Intercept                                        1.85      0.13     1.60
## phi_Intercept                                    2.62      0.11     2.40
## eval_period_normalized                           1.10      0.15     0.80
## treatmentdensity                                -0.51      0.19    -0.89
## treatmentdotplot                                -0.38      0.19    -0.73
## treatmenthops                                   -0.40      0.20    -0.80
## treatmenthopsdist                               -0.31      0.19    -0.68
## treatmentinterval                               -0.02      0.19    -0.38
## treatmentpoint                                   0.18      0.19    -0.19
## treatmenttable                                  -0.27      0.19    -0.65
## eval_period_normalized:treatmentdensity         -1.10      0.25    -1.60
## eval_period_normalized:treatmentdotplot         -0.24      0.26    -0.73
## eval_period_normalized:treatmenthops            -0.36      0.26    -0.89
## eval_period_normalized:treatmenthopsdist        -0.10      0.24    -0.57
## eval_period_normalized:treatmentinterval        -0.16      0.24    -0.63
## eval_period_normalized:treatmentpoint           -0.08      0.26    -0.58
## eval_period_normalized:treatmenttable           -0.55      0.26    -1.05
## phi_eval_period_normalized                       0.19      0.26    -0.33
## phi_treatmentdensity                            -0.55      0.16    -0.86
## phi_treatmentdotplot                            -0.86      0.15    -1.14
## phi_treatmenthops                               -0.65      0.16    -0.97
## phi_treatmenthopsdist                           -0.34      0.16    -0.66
## phi_treatmentinterval                           -0.21      0.16    -0.52
## phi_treatmentpoint                              -0.62      0.16    -0.93
## phi_treatmenttable                              -0.69      0.16    -1.01
## phi_eval_period_normalized:treatmentdensity     -0.20      0.43    -1.00
## phi_eval_period_normalized:treatmentdotplot     -0.18      0.40    -0.94
## phi_eval_period_normalized:treatmenthops        -0.36      0.43    -1.21
## phi_eval_period_normalized:treatmenthopsdist    -0.30      0.43    -1.15
## phi_eval_period_normalized:treatmentinterval    -0.02      0.41    -0.83
## phi_eval_period_normalized:treatmentpoint        0.64      0.38    -0.10
## phi_eval_period_normalized:treatmenttable       -0.02      0.43    -0.88
##                                              u-95% CI Rhat Bulk_ESS Tail_ESS
## Intercept                                        2.11 1.00      953     1796
## phi_Intercept                                    2.84 1.00     1993     3304
## eval_period_normalized                           1.39 1.00     2046     2732
## treatmentdensity                                -0.13 1.00     1126     1841
## treatmentdotplot                                -0.02 1.00     1198     2203
## treatmenthops                                   -0.01 1.00     1270     2416
## treatmenthopsdist                                0.07 1.00     1248     2304
## treatmentinterval                                0.34 1.00     1101     2130
## treatmentpoint                                   0.55 1.00     1328     2097
## treatmenttable                                   0.11 1.00     1390     2195
## eval_period_normalized:treatmentdensity         -0.61 1.00     2139     3062
## eval_period_normalized:treatmentdotplot          0.27 1.00     2537     3092
## eval_period_normalized:treatmenthops             0.16 1.00     2644     3168
## eval_period_normalized:treatmenthopsdist         0.36 1.00     2644     2984
## eval_period_normalized:treatmentinterval         0.30 1.00     2291     3127
## eval_period_normalized:treatmentpoint            0.42 1.00     2559     3424
## eval_period_normalized:treatmenttable           -0.03 1.00     2363     3181
## phi_eval_period_normalized                       0.71 1.00     2094     2700
## phi_treatmentdensity                            -0.24 1.00     2118     2977
## phi_treatmentdotplot                            -0.57 1.00     2228     3085
## phi_treatmenthops                               -0.33 1.00     2310     2934
## phi_treatmenthopsdist                           -0.03 1.00     2413     3142
## phi_treatmentinterval                            0.10 1.00     2330     3118
## phi_treatmentpoint                              -0.32 1.00     2461     2809
## phi_treatmenttable                              -0.38 1.00     2502     3227
## phi_eval_period_normalized:treatmentdensity      0.65 1.00     2154     3480
## phi_eval_period_normalized:treatmentdotplot      0.62 1.00     2019     3243
## phi_eval_period_normalized:treatmenthops         0.49 1.00     2457     3217
## phi_eval_period_normalized:treatmenthopsdist     0.51 1.00     2681     3386
## phi_eval_period_normalized:treatmentinterval     0.77 1.00     2801     3404
## phi_eval_period_normalized:treatmentpoint        1.39 1.00     2339     3261
## phi_eval_period_normalized:treatmenttable        0.82 1.00     2847     3386
## 
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
summary(mbeta2)
##  Family: beta 
##   Links: mu = logit; phi = log 
## Formula: dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1 | usertoken) 
##          phi ~ eval_period_normalized
##    Data: rq3 (Number of observations: 1386) 
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~usertoken (Number of levels: 198) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.63      0.04     0.55     0.72 1.00     2032     3026
## 
## Population-Level Effects: 
##                                          Estimate Est.Error l-95% CI u-95% CI
## Intercept                                    1.73      0.13     1.48     1.98
## phi_Intercept                                2.09      0.04     2.01     2.17
## eval_period_normalized                       0.98      0.17     0.65     1.33
## treatmentdensity                            -0.38      0.19    -0.75    -0.01
## treatmentdotplot                            -0.17      0.18    -0.53     0.18
## treatmenthops                               -0.26      0.20    -0.66     0.13
## treatmenthopsdist                           -0.24      0.19    -0.61     0.13
## treatmentinterval                            0.01      0.19    -0.36     0.37
## treatmentpoint                               0.31      0.18    -0.04     0.66
## treatmenttable                              -0.11      0.19    -0.48     0.25
## eval_period_normalized:treatmentdensity     -0.97      0.25    -1.46    -0.46
## eval_period_normalized:treatmentdotplot     -0.06      0.24    -0.52     0.41
## eval_period_normalized:treatmenthops        -0.20      0.26    -0.70     0.32
## eval_period_normalized:treatmenthopsdist     0.01      0.25    -0.49     0.49
## eval_period_normalized:treatmentinterval    -0.08      0.25    -0.58     0.42
## eval_period_normalized:treatmentpoint       -0.15      0.25    -0.63     0.33
## eval_period_normalized:treatmenttable       -0.44      0.25    -0.95     0.06
## phi_eval_period_normalized                   0.17      0.13    -0.08     0.42
##                                          Rhat Bulk_ESS Tail_ESS
## Intercept                                1.00     1312     2368
## phi_Intercept                            1.00     3180     3388
## eval_period_normalized                   1.00     2202     2820
## treatmentdensity                         1.00     1480     2432
## treatmentdotplot                         1.00     1336     2097
## treatmenthops                            1.00     1400     2491
## treatmenthopsdist                        1.00     1429     2356
## treatmentinterval                        1.00     1411     2573
## treatmentpoint                           1.00     1485     2119
## treatmenttable                           1.00     1348     2244
## eval_period_normalized:treatmentdensity  1.00     2323     3033
## eval_period_normalized:treatmentdotplot  1.00     2504     3125
## eval_period_normalized:treatmenthops     1.00     2589     3246
## eval_period_normalized:treatmenthopsdist 1.00     2557     3065
## eval_period_normalized:treatmentinterval 1.00     2311     3260
## eval_period_normalized:treatmentpoint    1.00     2465     3014
## eval_period_normalized:treatmenttable    1.00     2289     2892
## phi_eval_period_normalized               1.00     3426     3630
## 
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
summary(mbeta3)
##  Family: beta 
##   Links: mu = logit; phi = log 
## Formula: dv ~ eval_period_normalized + treatment + (1 | usertoken) 
##          phi ~ eval_period_normalized
##    Data: rq3 (Number of observations: 1386) 
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~usertoken (Number of levels: 198) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.63      0.04     0.55     0.71 1.00     2266     3120
## 
## Population-Level Effects: 
##                            Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept                      1.73      0.13     1.48     1.98 1.00     1442
## phi_Intercept                  2.07      0.04     1.99     2.16 1.00     3014
## eval_period_normalized         0.74      0.07     0.60     0.88 1.00     3111
## treatmentdensity              -0.37      0.20    -0.75     0.02 1.00     1485
## treatmentdotplot              -0.17      0.18    -0.53     0.18 1.00     1587
## treatmenthops                 -0.26      0.20    -0.64     0.13 1.00     1579
## treatmenthopsdist             -0.24      0.20    -0.62     0.15 1.00     1803
## treatmentinterval              0.01      0.19    -0.34     0.38 1.00     1683
## treatmentpoint                 0.30      0.18    -0.06     0.65 1.00     1674
## treatmenttable                -0.11      0.19    -0.50     0.25 1.00     1735
## phi_eval_period_normalized     0.16      0.13    -0.08     0.41 1.00     3377
##                            Tail_ESS
## Intercept                      2407
## phi_Intercept                  3186
## eval_period_normalized         3553
## treatmentdensity               2502
## treatmentdotplot               2747
## treatmenthops                  2613
## treatmenthopsdist              2419
## treatmentinterval              2659
## treatmentpoint                 2344
## treatmenttable                 2375
## phi_eval_period_normalized     3520
## 
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
summary(mbeta4)
##  Family: beta 
##   Links: mu = logit; phi = log 
## Formula: dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1 | usertoken) 
##          phi ~ eval_period_normalized * treatment
##    Data: rq3 (Number of observations: 1386) 
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~usertoken (Number of levels: 198) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.63      0.04     0.56     0.72 1.00     1919     2903
## 
## Population-Level Effects: 
##                                              Estimate Est.Error l-95% CI
## Intercept                                        1.89      0.14     1.63
## phi_Intercept                                    2.69      0.11     2.46
## eval_period_normalized                           1.21      0.17     0.88
## treatmentdensity                                -0.56      0.20    -0.96
## treatmentdotplot                                -0.43      0.19    -0.81
## treatmenthops                                   -0.47      0.21    -0.88
## treatmenthopsdist                               -0.37      0.20    -0.77
## treatmentinterval                               -0.06      0.20    -0.44
## treatmentpoint                                   0.12      0.20    -0.27
## treatmenttable                                  -0.32      0.20    -0.72
## eval_period_normalized:treatmentdensity         -1.27      0.26    -1.79
## eval_period_normalized:treatmentdotplot         -0.37      0.27    -0.90
## eval_period_normalized:treatmenthops            -0.50      0.27    -1.04
## eval_period_normalized:treatmenthopsdist        -0.22      0.26    -0.72
## eval_period_normalized:treatmentinterval        -0.28      0.25    -0.75
## eval_period_normalized:treatmentpoint           -0.18      0.28    -0.74
## eval_period_normalized:treatmenttable           -0.69      0.28    -1.25
## phi_eval_period_normalized                       0.27      0.38    -0.50
## phi_treatmentdensity                            -0.62      0.16    -0.94
## phi_treatmentdotplot                            -0.94      0.15    -1.23
## phi_treatmenthops                               -0.73      0.17    -1.07
## phi_treatmenthopsdist                           -0.41      0.17    -0.75
## phi_treatmentinterval                           -0.28      0.16    -0.60
## phi_treatmentpoint                              -0.70      0.16    -1.02
## phi_treatmenttable                              -0.76      0.16    -1.08
## phi_eval_period_normalized:treatmentdensity     -0.34      0.55    -1.42
## phi_eval_period_normalized:treatmentdotplot     -0.29      0.51    -1.28
## phi_eval_period_normalized:treatmenthops        -0.49      0.54    -1.53
## phi_eval_period_normalized:treatmenthopsdist    -0.40      0.53    -1.46
## phi_eval_period_normalized:treatmentinterval    -0.10      0.52    -1.09
## phi_eval_period_normalized:treatmentpoint        0.60      0.49    -0.38
## phi_eval_period_normalized:treatmenttable       -0.12      0.55    -1.17
##                                              u-95% CI Rhat Bulk_ESS Tail_ESS
## Intercept                                        2.16 1.00     1120     1849
## phi_Intercept                                    2.90 1.00     2144     2947
## eval_period_normalized                           1.54 1.00     2236     2871
## treatmentdensity                                -0.16 1.00     1450     2327
## treatmentdotplot                                -0.05 1.00     1355     2204
## treatmenthops                                   -0.06 1.00     1545     2295
## treatmenthopsdist                                0.03 1.00     1348     2015
## treatmentinterval                                0.31 1.00     1295     2340
## treatmentpoint                                   0.52 1.00     1426     2714
## treatmenttable                                   0.07 1.00     1548     2533
## eval_period_normalized:treatmentdensity         -0.76 1.00     2619     3341
## eval_period_normalized:treatmentdotplot          0.15 1.00     2452     3408
## eval_period_normalized:treatmenthops             0.03 1.00     2536     3355
## eval_period_normalized:treatmenthopsdist         0.27 1.00     2681     3267
## eval_period_normalized:treatmentinterval         0.23 1.00     2688     2813
## eval_period_normalized:treatmentpoint            0.37 1.00     2377     3025
## eval_period_normalized:treatmenttable           -0.14 1.00     2309     2939
## phi_eval_period_normalized                       0.99 1.00     1742     2647
## phi_treatmentdensity                            -0.31 1.00     2538     3216
## phi_treatmentdotplot                            -0.62 1.00     2359     2816
## phi_treatmenthops                               -0.42 1.00     2368     3352
## phi_treatmenthopsdist                           -0.08 1.00     2574     3387
## phi_treatmentinterval                            0.04 1.00     2242     3193
## phi_treatmentpoint                              -0.39 1.00     2355     2962
## phi_treatmenttable                              -0.43 1.00     2284     3140
## phi_eval_period_normalized:treatmentdensity      0.75 1.00     2019     2986
## phi_eval_period_normalized:treatmentdotplot      0.73 1.00     2046     2837
## phi_eval_period_normalized:treatmenthops         0.58 1.00     2183     3213
## phi_eval_period_normalized:treatmenthopsdist     0.63 1.00     2260     3108
## phi_eval_period_normalized:treatmentinterval     0.92 1.00     2081     3156
## phi_eval_period_normalized:treatmentpoint        1.55 1.00     2305     3132
## phi_eval_period_normalized:treatmenttable        0.96 1.00     2196     2882
## 
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
summary(mbeta5)
##  Family: beta 
##   Links: mu = logit; phi = log 
## Formula: dv ~ eval_period_normalized + treatment + eval_period_normalized * treatment + (1 | usertoken) 
##          phi ~ eval_period_normalized * treatment
##    Data: rq3 (Number of observations: 1386) 
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~usertoken (Number of levels: 198) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.64      0.04     0.56     0.72 1.00     1903     2275
## 
## Population-Level Effects: 
##                                              Estimate Est.Error l-95% CI
## Intercept                                        1.90      0.13     1.64
## phi_Intercept                                    2.69      0.11     2.47
## eval_period_normalized                           1.21      0.16     0.89
## treatmentdensity                                -0.56      0.20    -0.96
## treatmentdotplot                                -0.44      0.19    -0.82
## treatmenthops                                   -0.47      0.21    -0.88
## treatmenthopsdist                               -0.38      0.20    -0.77
## treatmentinterval                               -0.07      0.19    -0.45
## treatmentpoint                                   0.11      0.20    -0.28
## treatmenttable                                  -0.33      0.20    -0.72
## eval_period_normalized:treatmentdensity         -1.27      0.26    -1.78
## eval_period_normalized:treatmentdotplot         -0.37      0.27    -0.88
## eval_period_normalized:treatmenthops            -0.50      0.27    -1.05
## eval_period_normalized:treatmenthopsdist        -0.23      0.26    -0.75
## eval_period_normalized:treatmentinterval        -0.27      0.25    -0.77
## eval_period_normalized:treatmentpoint           -0.18      0.28    -0.72
## eval_period_normalized:treatmenttable           -0.70      0.28    -1.25
## phi_eval_period_normalized                       0.28      0.37    -0.44
## phi_treatmentdensity                            -0.63      0.16    -0.96
## phi_treatmentdotplot                            -0.94      0.15    -1.24
## phi_treatmenthops                               -0.74      0.17    -1.07
## phi_treatmenthopsdist                           -0.41      0.16    -0.74
## phi_treatmentinterval                           -0.29      0.16    -0.60
## phi_treatmentpoint                              -0.70      0.16    -1.01
## phi_treatmenttable                              -0.76      0.16    -1.08
## phi_eval_period_normalized:treatmentdensity     -0.34      0.54    -1.39
## phi_eval_period_normalized:treatmentdotplot     -0.30      0.49    -1.28
## phi_eval_period_normalized:treatmenthops        -0.49      0.53    -1.53
## phi_eval_period_normalized:treatmenthopsdist    -0.42      0.53    -1.47
## phi_eval_period_normalized:treatmentinterval    -0.10      0.50    -1.07
## phi_eval_period_normalized:treatmentpoint        0.58      0.49    -0.38
## phi_eval_period_normalized:treatmenttable       -0.13      0.53    -1.17
##                                              u-95% CI Rhat Bulk_ESS Tail_ESS
## Intercept                                        2.17 1.00     1024     1619
## phi_Intercept                                    2.90 1.00     2211     2977
## eval_period_normalized                           1.53 1.00     2064     2784
## treatmentdensity                                -0.17 1.00     1199     1933
## treatmentdotplot                                -0.08 1.00     1289     1992
## treatmenthops                                   -0.05 1.00     1182     1793
## treatmenthopsdist                                0.03 1.00     1241     1790
## treatmentinterval                                0.30 1.00     1279     1891
## treatmentpoint                                   0.49 1.00     1071     2336
## treatmenttable                                   0.06 1.01     1312     2401
## eval_period_normalized:treatmentdensity         -0.76 1.00     2553     3269
## eval_period_normalized:treatmentdotplot          0.15 1.00     2421     3115
## eval_period_normalized:treatmenthops             0.03 1.00     2502     3248
## eval_period_normalized:treatmenthopsdist         0.27 1.00     2391     3105
## eval_period_normalized:treatmentinterval         0.23 1.00     2598     3319
## eval_period_normalized:treatmentpoint            0.38 1.00     2505     2636
## eval_period_normalized:treatmenttable           -0.15 1.00     2328     2798
## phi_eval_period_normalized                       0.97 1.00     1461     2402
## phi_treatmentdensity                            -0.31 1.00     2389     2880
## phi_treatmentdotplot                            -0.64 1.00     2378     3064
## phi_treatmenthops                               -0.41 1.00     2443     2921
## phi_treatmenthopsdist                           -0.09 1.00     2300     3299
## phi_treatmentinterval                            0.04 1.00     2578     3030
## phi_treatmentpoint                              -0.38 1.00     2450     3419
## phi_treatmenttable                              -0.45 1.00     2269     3127
## phi_eval_period_normalized:treatmentdensity      0.74 1.00     2134     3033
## phi_eval_period_normalized:treatmentdotplot      0.68 1.00     1713     2866
## phi_eval_period_normalized:treatmenthops         0.55 1.00     2209     3066
## phi_eval_period_normalized:treatmenthopsdist     0.59 1.00     1784     2858
## phi_eval_period_normalized:treatmentinterval     0.89 1.00     1801     2978
## phi_eval_period_normalized:treatmentpoint        1.54 1.00     1921     2841
## phi_eval_period_normalized:treatmenttable        0.91 1.00     2220     2978
## 
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).
summary(mbeta6)
##  Family: beta 
##   Links: mu = logit; phi = log 
## Formula: dv ~ eval_period_normalized + treatment + (1 | usertoken) 
##          phi ~ eval_period_normalized
##    Data: rq3 (Number of observations: 1386) 
## Samples: 4 chains, each with iter = 4000; warmup = 2000; thin = 2;
##          total post-warmup samples = 4000
## 
## Group-Level Effects: 
## ~usertoken (Number of levels: 198) 
##               Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
## sd(Intercept)     0.63      0.04     0.55     0.72 1.00     2327     2536
## 
## Population-Level Effects: 
##                            Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS
## Intercept                      1.74      0.14     1.46     2.02 1.00     1416
## phi_Intercept                  2.07      0.04     1.99     2.15 1.00     3020
## eval_period_normalized         0.75      0.07     0.60     0.89 1.00     3047
## treatmentdensity              -0.40      0.20    -0.79     0.00 1.00     1526
## treatmentdotplot              -0.19      0.19    -0.56     0.18 1.00     1511
## treatmenthops                 -0.29      0.21    -0.69     0.13 1.00     1785
## treatmenthopsdist             -0.26      0.20    -0.65     0.14 1.00     1611
## treatmentinterval              0.01      0.20    -0.38     0.40 1.00     1508
## treatmentpoint                 0.30      0.20    -0.09     0.68 1.00     1539
## treatmenttable                -0.13      0.20    -0.53     0.27 1.00     1548
## phi_eval_period_normalized     0.17      0.13    -0.08     0.42 1.00     3605
##                            Tail_ESS
## Intercept                      2138
## phi_Intercept                  3372
## eval_period_normalized         3208
## treatmentdensity               2584
## treatmentdotplot               2322
## treatmenthops                  2503
## treatmenthopsdist              2568
## treatmentinterval              2635
## treatmentpoint                 2148
## treatmenttable                 2674
## phi_eval_period_normalized     3739
## 
## Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
## and Tail_ESS are effective sample size measures, and Rhat is the potential
## scale reduction factor on split chains (at convergence, Rhat = 1).

PPC across models

pp_check(mbeta)

pp_check(mbeta2)

pp_check(mbeta3)

pp_check(mbeta4)

pp_check(mbeta5)

pp_check(mbeta6)

Conditional Effects

conditional_effects(mbeta, "treatment") # conditions = data.frame(size = 1)

conditional_effects(mbeta2, "treatment") # conditions = data.frame(size = 1)

conditional_effects(mbeta3, "treatment") # conditions = data.frame(size = 1)

conditional_effects(mbeta4, "treatment") # conditions = data.frame(size = 1)

conditional_effects(mbeta5, "treatment") # conditions = data.frame(size = 1)

conditional_effects(mbeta6, "treatment") # conditions = data.frame(size = 1)

Mean PPC

#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta, type='stat', stat='mean')

#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))
#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta2, type='stat', stat='mean')

#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))
#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta3, type='stat', stat='mean')

#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))
#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta4, type='stat', stat='mean')

#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))
#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta5, type='stat', stat='mean')

#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))
#q75 <- function(y) quantile(y, 0.75)
pp_check(mbeta6, type='stat', stat='mean')

#+scale_x_continuous(breaks = seq(-0.5, 0.5, length.out = 7), labels = c("1", "5","10","15","20","25", "30"))
bayesplot::pp_check(mbeta, type = "violin_grouped", group = "treatment")

bayesplot::pp_check(mbeta, type = "error_hist")

Plots

eval_period_ends = rq3 %>%
  data_grid(
    treatment,
    eval_period_normalized = c(-0.467, .5) #c(-0.467, .5)  # because we normalized trial to be from -0.5 to 0.5
  ) %>%
  add_fitted_samples(mbeta, re_formula = NA, var = "mu", dpar = TRUE) %>%
  ungroup() %>%
  mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
  mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))


#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends %>%
  mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
  mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
  ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
  geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
    scale_fill_OkabeIto(alpha = 0.5) +
  geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
  cowplot::theme_minimal_vgrid()

eval_period_ends2 = rq3 %>%
  data_grid(
    treatment,
    eval_period_normalized = c(-0.467, .5) #c(-0.467, .5)  # because we normalized trial to be from -0.5 to 0.5
  ) %>%
  add_fitted_samples(mbeta2, re_formula = NA, var = "mu", dpar = TRUE) %>%
  ungroup() %>%
  mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
  mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))


#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends2 %>%
  mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
  mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
  ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
  geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
    scale_fill_OkabeIto(alpha = 0.5) +
  geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
  cowplot::theme_minimal_vgrid()

Notice how when we remove interaction density seems to be significant. This is because we’re incorrectly applying the population’s evaluation curve.

eval_period_ends3 = rq3 %>%
  data_grid(
    treatment,
    eval_period_normalized = c(-0.467, .5) #c(-0.467, .5)  # because we normalized trial to be from -0.5 to 0.5
  ) %>%
  add_fitted_samples(mbeta3, re_formula = NA, var = "mu", dpar = TRUE) %>%
  ungroup() %>%
  mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
  mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))


#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends3 %>%
  mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
  mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
  ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
  geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
    scale_fill_OkabeIto(alpha = 0.5) +
  geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
  cowplot::theme_minimal_vgrid()

eval_period_ends4 = rq3 %>%
  data_grid(
    treatment,
    eval_period_normalized = c(-0.467, .5) #c(-0.467, .5)  # because we normalized trial to be from -0.5 to 0.5
  ) %>%
  add_fitted_samples(mbeta4, re_formula = NA, var = "mu", dpar = TRUE) %>%
  ungroup() %>%
  mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
  mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))


#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends4 %>%
  mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
  mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
  ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
  geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
    scale_fill_OkabeIto(alpha = 0.5) +
  geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
  cowplot::theme_minimal_vgrid()

eval_period_ends5 = rq3 %>%
  data_grid(
    treatment,
    eval_period_normalized = c(-0.467, .5) #c(-0.467, .5)  # because we normalized trial to be from -0.5 to 0.5
  ) %>%
  add_fitted_samples(mbeta5, re_formula = NA, var = "mu", dpar = TRUE) %>%
  ungroup() %>%
  mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
  mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))


#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends5 %>%
  mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
  mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
  ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
  geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
    scale_fill_OkabeIto(alpha = 0.5) +
  geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
  cowplot::theme_minimal_vgrid()

Again, without interaction we’re having issues.

eval_period_ends6 = rq3 %>%
  data_grid(
    treatment,
    eval_period_normalized = c(-0.467, .5) #c(-0.467, .5)  # because we normalized trial to be from -0.5 to 0.5
  ) %>%
  add_fitted_samples(mbeta6, re_formula = NA, var = "mu", dpar = TRUE) %>%
  ungroup() %>%
  mutate(treatment = fct_rev(fct_relevel(treatment, vis_display_order))) %>%
  mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))


#Conditional means (for "average" person on "average" scenario) on last trial:
eval_period_ends6 %>%
  mutate(evalPeriod = if_else(eval_period_normalized==0.5,"30","1")) %>%
  mutate(evalPeriod = factor(evalPeriod, levels = c("30","1"))) %>%
  ggplot(aes(y = treatment, x = mu, group = evalPeriod, fill = evalPeriod)) +
  geom_halfeyeh(fun.data = median_qih, fatten.point = 1.3) +
    scale_fill_OkabeIto(alpha = 0.5) +
  geom_vline(xintercept = 1, linetype = "dashed", color = "black") +
  cowplot::theme_minimal_vgrid()

max_eval=30

mu_mean = rq3 %>%
  mutate(
    eval_period_normalized = ((evalPeriod - max_eval) / max_eval) + 0.5
  ) %>%
  group_by(treatment, eval_period_normalized) %>%
  summarise(mean=mean(dv))
fit_lines6 = rq3 %>%
  data_grid(
    treatment,
    eval_period_normalized = seq_range(eval_period_normalized, n = 20)
  ) %>%
  add_fitted_samples(mbeta6, re_formula = NA, var = "mu", dpar = TRUE) %>%
  ungroup()  %>%
  mutate(sd = sqrt(mu * (1 - mu) / (1 + phi)))

mu_mean6 = fit_lines6 %>%
  group_by(treatment, eval_period_normalized) %>%
  summarise(mu_mean=mean(mu))
fit_lines = rq3 %>%
  data_grid(
    treatment,
    eval_period_normalized = seq_range(eval_period_normalized, n = 20)
  ) %>%
  add_fitted_samples(mbeta, re_formula = NA, var = "mu", dpar = TRUE) %>%
  ungroup() 
  #mutate(vis = fct_relevel(vis, vis_display_order))

As we see here, the model is not doing well. We need the interaction effect.

scale_fill_fit_lines = scale_fill_manual(
  values = RColorBrewer::brewer.pal(4, "Greys")[-1], guide = guide_legend(reverse = TRUE)
)
fit_lines %>%
  ggplot(aes(x = eval_period_normalized, y = mu)) +
  stat_lineribbon(.prob = c(.95, .8, .5)) +
  geom_hline(yintercept = seq(.7, 1, by=.05), color="gray75", alpha = 0.5) +
  geom_line(aes(y = mean), data = mu_mean, color = "red", size = 1) +
  facet_grid(. ~ forcats::fct_relevel(treatment, vis_display_order)) +
  scale_fill_fit_lines 

  #fit_line_plot_settings

This is the actual model

scale_fill_fit_lines = scale_fill_manual(
  values = RColorBrewer::brewer.pal(4, "Greys")[-1], guide = guide_legend(reverse = TRUE)
)
fit_lines %>%
  ggplot(aes(x = eval_period_normalized, y = mu)) +
  stat_lineribbon(.prob = c(.95, .8, .5)) +
  geom_hline(yintercept = seq(.7, 1, by=.05), color="gray75", alpha = 0.5) +
  geom_line(aes(y = mean), data = mu_mean, color = "red", size = 1) +
  facet_grid(. ~ forcats::fct_relevel(treatment, vis_display_order)) +
  scale_fill_fit_lines 

  #fit_line_plot_settings